<html>
<head>
<title>DataTools.py</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<style type="text/css">
.s0 { color: #808080;}
.s1 { color: #a9b7c6;}
.s2 { color: #cc7832;}
.s3 { color: #6897bb;}
.s4 { color: #6a8759;}
</style>
</head>
<body bgcolor="#2b2b2b">
<table CELLSPACING=0 CELLPADDING=5 COLS=1 WIDTH="100%" BGCOLOR="#606060" >
<tr><td><center>
<font face="Arial, Helvetica" color="#000000">
DataTools.py</font>
</center></td></tr></table>
<pre><span class="s0"># -*- coding: UTF-8 -*-</span>
<span class="s2">import </span><span class="s1">os</span>
<span class="s2">import </span><span class="s1">numpy </span><span class="s2">as </span><span class="s1">np</span>
<span class="s2">import </span><span class="s1">matplotlib.pyplot </span><span class="s2">as </span><span class="s1">plt</span>
<span class="s2">from </span><span class="s1">keras.utils </span><span class="s2">import </span><span class="s1">Sequence</span>
<span class="s0"># from keras.preprocessing import sequence</span>

<span class="s0"># 计算振动数据标准差</span>
<span class="s2">def </span><span class="s1">imsdatasigma(path</span><span class="s2">, </span><span class="s1">channel=</span><span class="s3">0</span><span class="s1">):</span>
    <span class="s1">filelist = os.listdir(path)</span>
    <span class="s1">num_of_files = len(filelist)</span>
    <span class="s1">datasigmalist = []</span>
    <span class="s2">for </span><span class="s1">file </span><span class="s2">in </span><span class="s1">filelist:</span>
        <span class="s1">filepath = os.path.join(path</span><span class="s2">, </span><span class="s1">file)</span>
        <span class="s1">data = np.loadtxt(filepath)[:</span><span class="s2">, </span><span class="s1">channel]</span>
        <span class="s0"># sigma = np.sqrt(np.mean(np.square(data-data.mean())))</span>
        <span class="s1">sigma = np.std(data)</span>
        <span class="s1">datasigmalist.append(sigma)</span>
    <span class="s1">xlinespace = range(</span><span class="s3">0</span><span class="s2">, </span><span class="s1">len(datasigmalist))</span>
    <span class="s1">plt.plot(xlinespace</span><span class="s2">, </span><span class="s1">datasigmalist)</span>
    <span class="s1">plt.xlabel(</span><span class="s4">'TimeStep'</span><span class="s1">)</span>
    <span class="s1">plt.ylabel(</span><span class="s4">'sigma'</span><span class="s1">)</span>
    <span class="s1">plt.savefig(</span><span class="s4">'sigma graph'</span><span class="s1">)</span>
    <span class="s1">plt.show()</span>
    <span class="s2">return </span><span class="s1">datasigmalist</span>

<span class="s0">#</span>
<span class="s0"># def getdata(path, siglen=2000, chn=0, dataname='IMSdata.txt'):</span>
<span class="s0">#     filelist = os.listdir(path)</span>
<span class="s0">#     num_of_files = len(filelist)</span>
<span class="s0">#     IMSdata = []</span>
<span class="s0">#     for i in range(0, num_of_files) :</span>
<span class="s0">#         filepath = os.path.join(path, filelist[i])</span>
<span class="s0">#         filecontent = np.loadtxt(filepath)[:siglen, chn]</span>
<span class="s0">#         age = i/num_of_files</span>
<span class="s0">#         dataitem = [filecontent, age]</span>
<span class="s0">#     IMSdata.append(dataitem)</span>
<span class="s0">#     datadir = os.path.join(path, dataname)</span>
<span class="s0">#     np.save(datadir, IMSdata)</span>
<span class="s0">#     return datadir</span>


<span class="s0"># 获取数据文件列表</span>
<span class="s2">def </span><span class="s1">getIMSdatalist(path</span><span class="s2">, </span><span class="s1">step=</span><span class="s3">3</span><span class="s1">):</span>
    <span class="s1">filelist = os.listdir(path)</span>
    <span class="s1">num_of_files = len(filelist)</span>
    <span class="s2">assert </span><span class="s1">num_of_files &gt; step</span>
    <span class="s1">datalist = []</span>
    <span class="s2">for </span><span class="s1">i </span><span class="s2">in </span><span class="s1">range(</span><span class="s3">0</span><span class="s2">, </span><span class="s1">num_of_files-step+</span><span class="s3">1</span><span class="s1">) :</span>
        <span class="s1">dataitemlst = []</span>
        <span class="s2">for </span><span class="s1">j </span><span class="s2">in </span><span class="s1">range(</span><span class="s3">0</span><span class="s2">, </span><span class="s1">step):</span>
            <span class="s1">filepath = os.path.join(path</span><span class="s2">, </span><span class="s1">filelist[i+j])</span>
            <span class="s1">dataitemlst.append(filepath)</span>
        <span class="s1">dataitem = [dataitemlst</span><span class="s2">, </span><span class="s1">(i+step)/num_of_files]</span>
        <span class="s1">datalist.append(dataitem)</span>
    <span class="s0"># xlist = [row[0] for row in datalist]</span>
    <span class="s0"># ylist = [row[1] for row in datalist]</span>
    <span class="s2">return </span><span class="s1">datalist</span>


<span class="s2">class </span><span class="s1">IMSDataGenerator(Sequence):</span>
    <span class="s2">def </span><span class="s1">__init__(self</span><span class="s2">, </span><span class="s1">datalist</span><span class="s2">, </span><span class="s1">chn=</span><span class="s3">2</span><span class="s2">, </span><span class="s1">siglen=</span><span class="s3">2000</span><span class="s2">, </span><span class="s1">batchsize=</span><span class="s3">10</span><span class="s2">, </span><span class="s1">timestep=</span><span class="s3">3</span><span class="s2">, </span><span class="s1">shuffle=</span><span class="s2">True</span><span class="s1">):</span>
        <span class="s1">self.datalist = datalist</span>
        <span class="s1">self.chn = chn</span>
        <span class="s1">self.siglen = siglen</span>
        <span class="s1">self.batchsize = batchsize</span>
        <span class="s1">self.shuffle =shuffle</span>
        <span class="s1">self.indexes = np.arange(len(self.datalist))</span>
        <span class="s1">self.timestep = timestep</span>

    <span class="s2">def </span><span class="s1">__len__(self):</span>
        <span class="s2">return </span><span class="s1">int(np.floor(len(self.datalist)/self.batchsize))</span>

    <span class="s2">def </span><span class="s1">on_epoch_end(self):</span>
        <span class="s2">if </span><span class="s1">self.shuffle == </span><span class="s2">True</span><span class="s1">:</span>
            <span class="s1">np.random.shuffle(self.indexes)</span>

    <span class="s2">def </span><span class="s1">__getitem__(self</span><span class="s2">, </span><span class="s1">index):</span>
        <span class="s1">indexes = self.indexes[index*self.batchsize:(index+</span><span class="s3">1</span><span class="s1">)*self.batchsize]</span>
        <span class="s1">list_temp = [self.datalist[k] </span><span class="s2">for </span><span class="s1">k </span><span class="s2">in </span><span class="s1">indexes]</span>
        <span class="s1">x</span><span class="s2">, </span><span class="s1">y = self.__datageneration(list_temp)</span>
        <span class="s2">return </span><span class="s1">x</span><span class="s2">,</span><span class="s1">y</span>

    <span class="s2">def </span><span class="s1">__datageneration(self</span><span class="s2">, </span><span class="s1">indexes):</span>
        <span class="s1">x = []</span>
        <span class="s1">y = []</span>
        <span class="s2">for </span><span class="s1">block </span><span class="s2">in </span><span class="s1">indexes:</span>
            <span class="s1">y.append(block[</span><span class="s3">1</span><span class="s1">])</span>
            <span class="s1">xtemp=[]</span>
            <span class="s2">for </span><span class="s1">item </span><span class="s2">in </span><span class="s1">block[</span><span class="s3">0</span><span class="s1">]:</span>
                <span class="s1">datablock = np.loadtxt(item)[:self.siglen</span><span class="s2">, </span><span class="s1">self.chn]</span>
                <span class="s1">xtemp.append(datablock)</span>
            <span class="s1">x.append(xtemp)</span>
        <span class="s0"># x = x.reshape((self.siglen,1))</span>
        <span class="s1">x = np.array(x)</span>
        <span class="s1">x = x.reshape(self.batchsize</span><span class="s2">, </span><span class="s1">self.timestep</span><span class="s2">, </span><span class="s1">self.siglen</span><span class="s2">, </span><span class="s3">1</span><span class="s1">)</span>
        <span class="s0"># x = np.reshape(x, (x.shape[:], 1))</span>
        <span class="s1">y = np.array(y)</span>
        <span class="s1">y= y.reshape(self.batchsize</span><span class="s2">, </span><span class="s3">1</span><span class="s1">)</span>
        <span class="s2">return </span><span class="s1">x</span><span class="s2">, </span><span class="s1">y</span>


<span class="s2">def </span><span class="s1">ims_dataset_split(datalist</span><span class="s2">, </span><span class="s1">shuffle=</span><span class="s2">True, </span><span class="s1">ratio=</span><span class="s3">0.8 </span><span class="s1">):</span>
    <span class="s1">datasetidexes = np.arange(len(datalist))</span>
    <span class="s2">if </span><span class="s1">shuffle == </span><span class="s2">True</span><span class="s1">:</span>
        <span class="s1">np.random.shuffle(datasetidexes)</span>
    <span class="s1">train_set_indexes = datasetidexes[</span><span class="s3">0</span><span class="s1">:int(len(datalist)*ratio)]</span>
    <span class="s1">test_set_indexes = datasetidexes[int(len(datalist)*ratio):]</span>
    <span class="s0"># train_set_list = [datalist[i] for i in train_set_indexes]</span>
    <span class="s0"># test_set_list = [datalist[k] for k in test_set_indexes ]</span>
    <span class="s2">return </span><span class="s1">train_set_indexes</span><span class="s2">, </span><span class="s1">test_set_indexes</span>

<span class="s2">if </span><span class="s1">__name__ == </span><span class="s4">'__main__'</span><span class="s1">:</span>
    <span class="s0"># path = '../IMS/2nd_test/2nd_test'</span>
    <span class="s1">path = </span><span class="s4">'../IMS/3rd_test/4th_test/txt'</span>
    <span class="s1">data_sigma = imsdatasigma(path)</span>
    <span class="s0"># datalist = getIMSdatalist(path)</span>
    <span class="s0"># trainlist, test_list = ims_dataset_split(datalist)</span>
    <span class="s0"># testdataset = [datalist[i] for i in test_list]</span>
    <span class="s0"># # datalist[0]</span>
    <span class="s0"># # datalist[1]</span>
    <span class="s0"># # data = np.loadtxt(datalist[0][0][0])</span>
    <span class="s0"># # data1 = data[:, 0]</span>
    <span class="s0"># # data2 = data[:, 1]</span>
    <span class="s0"># imstrainingdatagenerator = IMSDataGenerator(testdataset)</span>
    <span class="s0"># x, y = imstrainingdatagenerator.getitem(3)</span>
    <span class="s0"># print(x.shape)</span>
    <span class="s0"># imstrainingdatagenerator.getitem(4)</span>
    <span class="s0"># xlist, ylist = getdatalist(path, step=3)</span>
    <span class="s0"># # print(datalist[5]['value'][0], datalist[5]['age'])</span>
    <span class="s0"># data = np.loadtxt(xlist[0][1])</span>
    <span class="s0"># y = ylist[0]</span>
    <span class="s0"># print(y)</span>
<span class="s0"># datadir = getdata(path)</span>




<span class="s1">np.reshape</span>
</pre>
</body>
</html>